/**
 * @file 一个爬虫类
 * @author fanxingguang
 */
// 引入模块
// let https = require('https');
'use strict';
let http = require('http');
// let fs = require('fs');
// let path = require('path');
let cheerio = require('cheerio');
let util = require('./util');

let getHtml = url => {
    console.log('spider: ', url);
    let options = util.parseUrl(url);
    options.port = 80;
    return new Promise((resolve, reject) => {
        http.get(options, res => {
            let html = '';
            res.setEncoding('utf-8');
            res.on('data', chunk => {
                html += chunk;
            });
            res.on('end', () => {
                resolve(cheerio.load(html, {decodeEntities: false}));
            });
        }).on('error', function (err) {
            reject(err);
        });
    });
};
// options = {
//     listUrl: '',
//     listSelector: '',
//     page: [{
//         key: 'title',
//         selector: '',
//         type: 'text'
//     }]
// }
let spider = options => {
    let resultList = [];
    let promiseList = [];
    let listUrl = options.listUrl;
    return new Promise((resolve, reject) => {
        getHtml(listUrl).then(($) => {
            $(options.listSelector).each((index, node) => {
                let href = $(node).attr('href');
                // 如果不是以http开头
                if (!(/^http:\/\//.test(href))) {
                    href = util.parseUrl(listUrl).hostname + href;
                }
                let promise = getHtml(href).then($ => {
                    let result = {};
                    for (let item of options.page) {
                        if (item.type === 'text' || item.type === undefined) {
                            result[item.key] = $(item.selector).text();
                        }
                        else {
                            result[item.key] = $(item.selector).html();
                        }
                    }
                    resultList.push(result);
                });
                promiseList.push(promise);
            });
            Promise.all(promiseList).then(() => {
                resolve(resultList);
            });
        });
    });
};

module.exports = spider;

